#!/bin/bash
#set -x
set -e

source ./config/config
source ./script/lib.sh


## 检查pod运行状态

echo -e "\033[93m|+++++++++++++++++++++++++++++++++++++++++++++++++++++++++|"
echo -e "|                        pod                     |"
echo -e "|+++++++++++++++++++++++++++++++++++++++++++++++++++++++++|\033[0m"

echo -e "#################  事件检查  #################"
pod_events_exception=`kubectl get events -A |grep -v Normal`

if [[ -n $pod_events_exception ]] ;then
	wResult "WARNING" pod_events_exception
else
	wResult "NORMAL" pod_events_exception
fi

wDetail "$pod_events_exception" pod_events_exception

echo -e "#################  核心组件检查  #################"
echo -e "kube-apiserver status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{job="apiserver"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "---------------------\n" >>$result_dir/pod_core_exception_detail

echo -e "kube-controller status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{job="kube-controller-manager"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "---------------------\n" >>$result_dir/pod_core_exception_detail

echo -e "kube-scheduler status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{job="kube-scheduler"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "---------------------\n" >>$result_dir/pod_core_exception_detail

echo -e "etcd status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{job="kube-etcd"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "etcd leader status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=etcd_server_is_leader' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "---------------------\n" >>$result_dir/pod_core_exception_detail

echo -e "kubelet status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{component="kubelet",job="kubelet"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "---------------------\n" >>$result_dir/pod_core_exception_detail

echo -e "kube-proxy status：" >>$result_dir/pod_core_exception_detail
pod_core_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=up{job="kube-proxy"}'| jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"'`

pod_core_kube_apiserver_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                                   --data 'query=up{job="kube-controller-manager"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

pod_core_kube_controller_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                                    --data 'query=up{job="kube-controller-manager"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

pod_core_kube_scheduler_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                                   --data 'query=up{job="kube-scheduler"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

pod_core_etcd_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                         --data 'query=up{job="kube-etcd"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

pod_core_etcd_leader_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                         --data 'query=sum(etcd_server_is_leader)' | jq -r '.data.result.[] | "\(.value.[1])"'`

pod_core_kubelet_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                            --data 'query=up{component="kubelet",job="kubelet"}' | jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

pod_core_kube_proxy_exception=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                               --data 'query=up{job="kube-proxy"}'| jq -r '.data.result.[] | "\(.metric.job) \(.metric.instance) \(.value.[1])"' | awk '{if($3=0) print $1}'`

if [[ -n $pod_core_kube_apiserver_exception ]] || [[ -n $pod_core_kube_controller_exception ]] || [[ -n $pod_core_kube_scheduler_exception ]] || [[ -n $pod_core_etcd_exception ]] || [[ $pod_core_etcd_leader_exception -ne "1" ]] || [[ -n $pod_core_kubelet_exception ]] || [[ -n $pod_core_kube_proxy_exception ]];then
	wResult "FATAL" pod_core_exception
else
	wResult "NORMAL" pod_core_exception
fi
wDetail "$pod_core_exception\n" pod_core_exception

echo -e "#################  pod健康状态  #################"
pod_health_exception_heading="pod名称:运行状态:正常状态\n"
pod_health_exception=`kubectl get pods --template=checkpods.tpl -o go-template-file -A  |grep -v true |grep -v Succeeded |grep -v check`
#echo -e "pod名称:运行状态:正常状态" >>$result_dir/pod_health_exception_detail

if [[ -n $pod_health_exception ]];then
	wResult "FATAL" pod_health_exception
	pod_health_exception=$pod_health_exception_heading$pod_health_exception
	wDetail_column "$pod_health_exception" pod_health_exception ":"
else
	wResult "NORMAL" pod_health_exception
	wDetail "当前没有运行异常的pod" pod_health_exception
fi

echo -e "#################  pod的资源状况  #################"
#pod_resource_status=`kubectl top pods -A --containers | sort --reverse --key 5 --numeric`
host=`kubectl get node | awk '{print $1}' | grep -v NAME`
pod_resource_status=`  for item in $host; do
                       echo -e "$item 资源状况为:"
                         kubectl get pods --all-namespaces --field-selector spec.nodeName=$item -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' | while read -r namespace pod; do kubectl top pod "$pod" -n "$namespace" --no-headers 2>/dev/null; done | awk '
                       BEGIN {print "NAMESPACE", "NAME", "CPU(cores)", "MEMORY(GiB)"}
                       {
                           split($3, mem, "Mi");
                           memory_gib = mem[1] / 1024;
                           printf "%-20s %-50s %-10s %.2fGiB\n", $1, $2, $3, memory_gib;
                           total += memory_gib;
                       }
                       END {printf "\nTOTAL MEMORY: %.2fGiB\n", total}' | column -t
                       echo -e "\r\r"
                       done`
wDetail "$pod_resource_status" pod_resource_status

echo -e "#################  cpu利用率检查  #################"
pod_cpu_status_heading="命名空间 pod名称 百分比\n"
pod_cpu_status=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=topk(1000,sum (rate (container_cpu_usage_seconds_total{image!="",pod!~".*tunnelserver.*"}[5m])) by( container, namespace, pod) / (sum (container_spec_cpu_quota{pod!~".*tunnelserver.*"}) by(container, namespace, pod) /100000) * 100)' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | awk '{printf "%s\t%s\t%.2f\n", $1,  $2, $3}' | awk '{ $3 = $3 "%" } { print }' | column -t `

pod_cpu_status=$pod_cpu_status_heading$pod_cpu_status

pod_cpu_status_fatal=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                       --data 'query=topk(1000,sum (rate (container_cpu_usage_seconds_total{image!="",pod!~".*tunnelserver.*"}[5m])) by( container, namespace, pod) / (sum (container_spec_cpu_quota{pod!~".*tunnelserver.*"}) by(container, namespace, pod) /100000) * 100)' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | awk '{if($3>90) print $1}'`
pod_cpu_status_warning=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                         --data 'query=topk(1000,sum (rate (container_cpu_usage_seconds_total{image!="",pod!~".*tunnelserver.*"}[5m])) by( container, namespace, pod) / (sum (container_spec_cpu_quota{pod!~".*tunnelserver.*"}) by(container, namespace, pod) /100000) * 100)' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | awk '{if($3>80) print $1}'`
if [[ -n $pod_cpu_status_fatal ]];then
	wResult "FATAL" pod_cpu_status
elif [[ -n $pod_cpu_status_warning ]];then
	wResult "WARNING" pod_cpu_status
else
	wResult "NORMAL" pod_cpu_status
fi
wDetail_column "$pod_cpu_status" pod_cpu_status

echo -e "#################  内存利用率检查  #################"
pod_memory_status_heading="命名空间 pod名称 百分比\n"
pod_memory_status=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=topk(1000,sum by(container, namespace, pod) (container_memory_rss{image!="",pod!~".*tunnelserver.*"})/ sum by(container, namespace, pod) (container_spec_memory_limit_bytes{image!="",pod!~".*tunnelserver.*"})* 100 != +Inf )' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | grep -v NaN | awk '{printf "%s\t%s\t%.2f\n", $1,  $2, $3}' | awk '{ $3 = $3 "%" } { print }' | column -t `

pod_memory_status=$pod_memory_status_heading$pod_memory_status

pod_memory_status_fatal=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                          --data 'query=topk(1000,sum by(container, namespace, pod) (container_memory_rss{image!="",pod!~".*tunnelserver.*"})/ sum by(container, namespace, pod) (container_spec_memory_limit_bytes{image!="",pod!~".*tunnelserver.*"})* 100 != +Inf )' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | awk '{if($3>90) print $1}'`
pod_memory_status_warning=`curl -s ${PROMETHEUS_URL}/api/v1/query\
                            --data 'query=topk(1000,sum by(container, namespace, pod) (container_memory_rss{image!="",pod!~".*tunnelserver.*"})/ sum by(container, namespace, pod) (container_spec_memory_limit_bytes{image!="",pod!~".*tunnelserver.*"})* 100 != +Inf )' | jq -r '.data.result.[] | "\(.metric.namespace) \(.metric.pod) \(.value.[1])"' | awk '{if($3>80) print $1}'`
if [[ -n $pod_memory_status_fatal ]];then
	wResult "FATAL" pod_memory_status
elif [[ -n $pod_memory_status_warning ]];then
	wResult "WARNING" pod_memory_status
else
	wResult "NORMAL" pod_memory_status
fi
wDetail_column "$pod_memory_status" pod_memory_status

echo -e "#################  自动重启状态检查  #################"
pod_restart_status_heading="命名空间 pod名称\n"
pod_restart_status=`curl -s ${PROMETHEUS_URL}/api/v1/query\
 --data 'query=rate(kube_pod_container_status_restarts_total{job="kube-state-metrics",namespace=~".*"}[1d]) > 0 ' | jq -r '.data.result.[]| "\(.metric.namespace) \(.metric.pod)"'`

pod_restart_status=$pod_restart_status_heading$pod_restart_status

if [[ -n $pod_restart_status ]];then
	wResult "FATAL" pod_restart_status
else
	wResult "NORMAL" pod_restart_status
fi
wDetail_column "$pod_restart_status" pod_restart_status
